package com.github.search.spider;

import com.github.search.model.Article;
import com.github.search.repository.ArticleRepository;
import org.elasticsearch.common.UUIDs;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class SegmentfaultProcesser implements PageProcessor {

    public SegmentfaultProcesser(ArticleRepository articleRepository) {
        this.articleRepository = articleRepository;
    }

    ArticleRepository articleRepository;

    public static String url = "https://segmentfault.com/questions?page=1";

    @Override
    public void process(Page page) {
        if (page.getUrl().get().contains("/q/")) {
            //详情页
            Article article = new Article();
            article.setId((int) ((Math.random() * 9 + 1) * 100000));
            article.setSid(UUIDs.base64UUID());
            article.setTitle(page.getHtml().xpath("/html/head/title/text()").get());
//            System.out.println(page.getHtml().xpath("/html/head/title/text()").get());
            article.setUrl(page.getUrl().get());
            article.setContent(page.getHtml().xpath("/html/body/div[3]/div[4]/div/div[1]").get());
//            System.out.println(article.getContent());
            Article article1 = articleRepository.save(article);
        } else if (page.getUrl().get().contains("questions?page=")) {
            //提取详情页
            page.addTargetRequests(page.getHtml().xpath("/html/body/div[3]/div[2]/div/div[1]/div[2]").links().all());
            //提取下一页
            page.addTargetRequests(page.getHtml().xpath("/html/body/div[3]/div[2]/div/div[1]/div[3]").links().all());
        }
    }

    @Override
    public Site getSite() {
        return Site.me().setDomain("https://segmentfault.com").setRetryTimes(8).setSleepTime(2000);
    }


}
